# _*_ coding:utf-8 _*_
# @Time  : 2022.10.11
# @Author: zizlee
import json
import datetime
import pathlib
import time

import requests
import pandas as pd
import urllib3
from urllib3 import disable_warnings
urllib3.disable_warnings()

""" CZC的交割数据 """

# SERVER_API = "http://127.0.0.1:8000/v1/"
SERVER_API = "https://210.13.218.130:9000/v1/"

VARIETY = {
    '棉花': 'CF',
    '干制红枣': 'CJ',
    '棉纱': 'CY',
    '平板玻璃': 'FG',
    '甲醇': 'MA',
    '菜籽油': 'OI',
    '菜籽粕': 'RM',
    '纯碱': 'SA',
    '硅铁': 'SF',
    '锰硅': 'SM',
    '白砂糖': 'SR',
    '精对苯二甲酸': 'TA',
    '尿素': 'UR',
    '硬麦': 'WT',  # 没做
    '强麦': 'WS',  # 没做
    '早籼稻': 'RI',  # 没做
    '晚籼稻': 'LR',  # 没做
    '普通小麦': 'PM',  # 没做
    '优质强筋小麦': 'WH',  # 没做
    '油菜籽': 'RS',  # 没做
    '动力煤': 'ZC',  # 20140131
    '粳稻谷': 'JR',  # 20140331
    '鲜苹果': 'AP',  # 20180531
    '涤纶短纤': 'PF',  # 20210531
    '花生仁': 'PK',  # 20211030

}


def _get_variety_en(variety_name):
    en = VARIETY.get(variety_name)
    if not en:
        raise ValueError('{}的代码不存在,请补充！'.format(variety_name))
    return en


def spider_delivery_file(delivery_date: datetime.date):
    # 月交割数据
    delivery_url = "http://www.czce.com.cn/cn/DFSStaticFiles/Future/{}/{}/FutureDataSettlematched.xls".format(
        delivery_date.year, delivery_date.strftime('%Y%m%d'))
    # 保存为当月的最后一天
    if delivery_date.month == 12:
        last_day = datetime.date(delivery_date.year, delivery_date.month, 31)
    else:
        last_day = datetime.date(delivery_date.year, delivery_date.month + 1, 1) + datetime.timedelta(days=-1)

    delivery_file = pathlib.Path("czce/delivery/{}.xls".format(last_day.strftime('%Y%m%d')))

    headers = {'User-Agent': "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) "
                             "Chrome/78.0.3904.108 Safari/537.36",
               'Host': 'www.czce.com.cn'}

    r = requests.get(delivery_url, headers=headers)
    with open(delivery_file, 'wb') as fp:
        fp.write(r.content)
    print('获取{}_{}.CZC月交割数据数据完成！'.format(delivery_date.strftime('%Y%m%d'), last_day.strftime('%Y%m%d')))


def parse_delivery_file(delivery_date: datetime.date):
    quote_date_string = delivery_date.strftime('%Y%m%d')
    # 2017.11及以前的数据为html
    if delivery_date.strftime('%Y%m%d') <= '20100731':
        file = pathlib.Path('czce/delivery/{}.html'.format(quote_date_string[:6]))
        if not file.exists():
            print("没有发现{}.CZC的交割文件,请先抓取数据!".format(quote_date_string))
            return
        xls_df = pd.read_html(file, encoding='utf8', skiprows=[])[1]
        xls_df.dropna(inplace=True)
        if xls_df.columns.tolist() != ['品种', '交割数量', '交割额（元）']:
            print('月仓单数据格式有误,请检查！', xls_df.columns.tolist())
            return
        xls_df.rename(columns={'品种': 'variety_name', '交割数量': 'volume', '交割额（元）': 'amount'}, inplace=True)
        flags = []
        f = 0
        for v in xls_df['variety_name'].tolist():
            if v == '合计':
                f = 1
            flags.append(f)
        xls_df['flag'] = flags
        # 去掉合计以后的行
        xls_df = xls_df[xls_df['flag'] == 0]
        del xls_df['flag']

    elif '20100801' <= quote_date_string <= '20151130':
        file = pathlib.Path('czce/delivery/{}.txt'.format(quote_date_string[:6]))
        if not file.exists():
            print("没有发现{}.CZC的交割文件,请先抓取数据!".format(quote_date_string))
            return
        with open(file, encoding='gbk') as f:
            lines = f.readlines()
        lines.pop(0)
        lines = [row.strip().split(',') for row in lines if not row.startswith('合计')]
        xls_df = pd.DataFrame(lines, columns=['variety_name', 'volume', 'amount'])
    else:
        file = pathlib.Path('czce/delivery/{}.xls'.format(quote_date_string))
        if not file.exists():
            print("没有发现{}.CZC的交割文件,请先抓取数据!".format(quote_date_string))
            return
        xls_df = pd.read_excel(file, skiprows=[0], thousands=',')
        if xls_df.columns.tolist() != ['品种', '实物交割数量', '实物交割额（元）']:
            print('月仓单数据格式有误,请检查！', xls_df.columns.tolist())
            return

        xls_df.rename(columns={'品种': 'variety_name', '实物交割数量': 'volume', '实物交割额（元）': 'amount'}, inplace=True)
        xls_df = xls_df[xls_df['variety_name'] != '合计']
    xls_df['variety_name'] = xls_df['variety_name'].apply(lambda x: x.strip())
    xls_df['variety_en'] = xls_df['variety_name'].apply(lambda x: _get_variety_en(x))
    xls_df['delivery_date'] = quote_date_string
    xls_df['volume_unit'] = '张'
    xls_df['amount_unit'] = '元'
    delivery_data = xls_df.to_dict(orient='records')
    print('----- 解析{}.CZC交割文件成功,数量:{} -----'.format(quote_date_string, len(delivery_data)))
    if len(delivery_data) > 0:
        folder = pathlib.Path("resolution/{}/".format(quote_date_string))
        print(folder)
        if not folder.exists():
            folder.mkdir(parents=True)
        resolution_delivery_file = pathlib.Path("resolution/{}/CZC_Delivery.json".format(quote_date_string))
        # 保存到本地文件
        with open(resolution_delivery_file, 'w', encoding='utf8') as fp:
            json.dump(delivery_data, fp, indent=2, ensure_ascii=False)


def save_delivery_to_server(delivery_date: datetime.date):
    quote_date_string = delivery_date.strftime('%Y%m%d')
    resolution_delivery_file = pathlib.Path("resolution/{}/CZC_Delivery.json".format(quote_date_string))
    if not resolution_delivery_file.exists():
        raise ValueError('{}.CZC行情数据文件不存在,请先解析保存！'.format(quote_date_string))
    # 读取本地文件
    with open(resolution_delivery_file, 'r', encoding='utf8') as fp:
        json_data = json.load(fp)
    # 保存到目标API
    save_api = SERVER_API + 'exchange/delivery/'
    r = requests.post(url=save_api, json=json_data, verify=False)
    print(r.json())


def spider_201008_201511(delivery_date: datetime.date):
    quote_date_string = delivery_date.strftime('%Y%m%d')
    delivery_url = 'http://www.czce.com.cn/cn/exchange/{}/datasettlematched/{}.txt'.format(delivery_date.year, quote_date_string[:6])
    headers = {'User-Agent': "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) "
                             "Chrome/78.0.3904.108 Safari/537.36",
               'Host': 'www.czce.com.cn'}
    delivery_file = pathlib.Path("czce/delivery/{}.txt".format(delivery_date.strftime('%Y%m')))
    r = requests.get(delivery_url, headers=headers)
    with open(delivery_file, 'wb') as fp:
        fp.write(r.content)
    print('获取{}.CZC月交割数据数据完成！'.format(delivery_date.strftime('%Y%m%d')))


if __name__ == '__main__':
    # 从2005.03月开始，是html 2005.03-2010.07
    # 2010.08-2015.11，使用txt
    # 2015.12-  xls

    start = datetime.date(2023, 11, 1)
    end = datetime.date.today()
    business_days_rng = pd.date_range(start, end, freq='BM')
    for d in business_days_rng:
        spider_delivery_file(delivery_date=d)  # 爬取文件
        # 解析数据
        if d.month == 12:
            month_last = datetime.date(d.year, d.month, 31)
        else:
            month_last = datetime.date(d.year, d.month + 1, 1) + datetime.timedelta(days=-1)
        parse_delivery_file(delivery_date=month_last)  # 解析文件
        save_delivery_to_server(delivery_date=month_last)  # 保存数据
        time.sleep(1)
